# -*- coding: utf-8 -*-
import scrapy,random,json,jsonpath,math
from xhs_spider.phone_agent import agent
from xhs_spider.items import XhsSpiderItem,NoteItem
import re
from xhs_spider.redisfilter import BloomFilter
from xhs_spider.randomseed import Randomuser
class XhsSpider(scrapy.Spider):
    name = 'xhs'
    custom_settings = {
        'DEFAULT_REQUEST_HEADERS':{
            "User-Agent": random.choice(agent),
            "Host":"www.xiaohongshu.com",
        }
    }
    # 粉丝连接
    fans_url = 'https://www.xiaohongshu.com/api/sns/v1/user/%s/followers?platform=Android&deviceId=cd791be3-2df0-3f0c-a137-9b0a4c8a796a&device_fingerprint=201809121411034abaed68357e498f045ae5b48365dfea019f44c63e626860&device_fingerprint1=201809121411034abaed68357e498f045ae5b48365dfea019f44c63e626860&versionName=5.24.1&channel=xiaohongshu&sid=session.1221344937852575358&lang=zh-Hans&t=1536849069&sign=2c7bc3579339638368dbad7fa3bab470'
    fllow = 'https://www.xiaohongshu.com/api/sns/v1/user/561e0c65a75c9555e1d8d793/followings?platform=Android&deviceId=cd791be3-2df0-3f0c-a137-9b0a4c8a796a&device_fingerprint=201809121411034abaed68357e498f045ae5b48365dfea019f44c63e626860&device_fingerprint1=201809121411034abaed68357e498f045ae5b48365dfea019f44c63e626860&versionName=5.24.1&channel=xiaohongshu&sid=session.1221344937852575358&lang=zh-Hans&t=1536849124&sign=6f6c6886380967366596e6e055b00786'

    info_url = 'https://www.xiaohongshu.com/api/sns/v3/user/561e0c65a75c9555e1d8d793/info?platform=Android&deviceId=cd791be3-2df0-3f0c-a137-9b0a4c8a796a&device_fingerprint=201809121411034abaed68357e498f045ae5b48365dfea019f44c63e626860&device_fingerprint1=201809121411034abaed68357e498f045ae5b48365dfea019f44c63e626860&versionName=5.24.1&channel=xiaohongshu&sid=session.1221338660721524020&lang=zh-Hans&t=1536843174&sign=9f6675644e1f141b94ce62911db4d7f6'
    def start_requests(self):
        # 种子用户
        ud=Randomuser()
        uid =ud.randomuserid()
        # 种子选手粉丝
        base_url = self.fans_url % uid
        # 种子选手关注
        fllow= self.fllow % uid
        yield scrapy.Request(base_url,callback=self.fans_list)
        yield scrapy.Request(fllow,callback=self.fans_list)
    # 粉丝列表
    def fans_list(self,response):
        html=response.text
        fans_list=json.loads(html).get('data')
        for fans in fans_list:
            # 构建新的请求使用
            userid = fans.get('userid')
            info_urls = self.info_url % userid
            # bf = BloomFilter()
            # if bf.isContains(info_urls):  # 判断字符串是否存在
            #     print('exists!')
            # else:
            #     print('not exists!')
            #     bf.insert(info_urls)
            yield scrapy.Request(info_urls,callback=self.fans_info)
    # 粉丝详情
    def fans_info(self,response):
        item=XhsSpiderItem()
        fans_html=response.text
        user = json.loads(fans_html).get('data')
        if user != '':
            def remove_emoji(desstr, restr=''):
                '''
                过滤表情
                '''
                try:
                    co = re.compile(u'[\U00010000-\U0010ffff]')
                except re.error:
                    co = re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]')
                return co.sub(restr, desstr)
            # 红书昵称
            item["nickname"] = remove_emoji(user.get('nickname',''))
            # 红书ID
            item["red_id"] = user.get('red_id')
            # 粉丝
            item["fans"] = user.get('fans','0')
            # 关注
            item["follows"] = user.get('follows','0')
            # 描述
            item["describ"] = remove_emoji(user.get('desc',''))
            # 性别
            gender =user.get('gender','')
            item['gender']= '女' if gender==1 else '男'
            # 收藏和获赞
            collected = user.get('collected',0)
            liked = user.get('liked',0)
            item['collect_like'] = int(collected) + int(liked)
            # 笔记数
            item['ndiscovery'] = user.get('ndiscovery',0)
            # 个人log
            item["user_logo"] = user.get('mini_program_info').get('thumb','')

            userid=user.get('userid')
            item['userid'] = user.get('userid')
            # 个人的空间地址
            item['user_url'] = 'https://www.xiaohongshu.com/user/profile/{}'.format(userid)

            # 如果粉丝大于2000,才抓取,过滤掉没有价值的信息
            if item['fans'] >= 2000:
                print('-----该账号粉丝数量大于2000,正在抓取------')
                yield item
            # 获得笔记的页数
                if item['ndiscovery'] != 0:
                    page_nums = int(math.ceil(item['ndiscovery']/10))
                    # 请求笔记的连接
                    for page in range(1,page_nums+1):
                        if page == 10:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=10&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536831356&sign=b5b61bf31080b569e5fade43a6555b18'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})
                        if page == 9:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=9&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536831248&sign=e6d28845fb4dcaf4a9bc95dcf3fc7d2a'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})
                        if page == 8:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=8&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536831104&sign=582e097459416cf370bace9c4bb1bfc9'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})
                        if page == 7:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=7&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536830996&sign=c9d3e5420a692cd083f343fea6999aa7'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})
                        if page == 6:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=6&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536830869&sign=b65dc563db5901ce39e7d334a0fe8e12'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})
                        if page == 5:
                            node_url ='https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=5&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536829400&sign=76ddb8f6a06303c9b7219d66dbefff19'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})
                        if page == 4:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=4&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536829313&sign=48f5acb1681d669adab2c53bcf5918be'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})
                        if page == 3:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=3&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536829236&sign=01176709154ebd85f543d625f4a62eac'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})
                        if page == 2:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=2&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536819388&sign=f85441d08b86858607ee3d2227dda007'.format(userid=userid)
                            yield scrapy.Request(node_url,callback=self.video_info,meta={'item':item})
                        if page == 1:
                            node_url = 'https://www.xiaohongshu.com/api/sns/v2/note/user/{userid}?page=1&page_size=10&sub_tag_id=&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536819748&sign=4d8edb2dfb564775175876e36cae2ac1'.format(userid=userid)
                            yield scrapy.Request(node_url, callback=self.video_info,meta={'item':item})


    # 视频详情
    def video_info(self,response):
        item = response.meta['item']
        notes_list = json.loads(response.text)
        note_data_list = notes_list.get('data').get('notes')
        for note in note_data_list:
            noteitem = NoteItem()
            noteitem['note_userid'] = item['userid']
            noteitem["note_id"] = note.get('id')
            def remove_emoji(desstr, restr=''):
                '''
                过滤表情
                '''
                try:
                    co = re.compile(u'[\U00010000-\U0010ffff]')
                except re.error:
                    co = re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]')
                return co.sub(restr, desstr)
            noteitem["note_title"] = remove_emoji(note.get('title'))

            type = note.get('type')


            if type == 'normal':

                headers = {
                    'X-Tingyun-Id': 'LbxHzUNcfig;c=2;r=438478276;u=88b60e7cc809ecbf57b64f848623279e12fa3062165580757abc76487485da7bde58644b58a914efd5ad91b8f8b55b5d::E1D3040A905637B0',
                    'Authorization': 'session.1221337646624184161',
                    'shield':'',
                    'device_id': '16885501-8fd1-3102-8dab-ba819586d3e5',
                }
                # 点赞链接
                comment_url = 'https://www.xiaohongshu.com/api/sns/v9/note/{noteid}?filter=profile_feed&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221230684757403642&lang=zh-Hans&t=1536833612&sign=392a66521e405f07c77303b2f295b59e'.format(noteid=noteitem["note_id"])
                # 评论数链接
                comment_nums_url = 'https://www.xiaohongshu.com/api/sns/v5/note/{note_id}/comment/list?start=&num=3&show_priority_sub_comments=1&platform=Android&deviceId=16885501-8fd1-3102-8dab-ba819586d3e5&device_fingerprint=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&device_fingerprint1=201809121444167d03fd896c92728373674fa3e2f8381b0144ee7e60e4a2d3&versionName=5.24.1&channel=xiaohongshu&sid=session.1221337646624184161&lang=zh-Hans&t=1536841691&sign=254131d787ed1b06af634093b5c90ef3'.format(note_id=noteitem["note_id"])
                yield scrapy.Request(comment_url,headers=headers,callback=self.comment_info,meta={'noteitem':noteitem,'item':item})
                yield scrapy.Request(comment_nums_url,callback=self.comment_num,meta={'noteitem':noteitem})
    # 点赞详情
    def comment_info(self,response):

        noteitem = response.meta['noteitem']
        item = response.meta['item']
        userid = item['userid']
        comment_list = json.loads(response.text)
        comment_data = comment_list.get('data')
        # 收藏数
        noteitem["fav_count"] = comment_data.get('fav_count',0)
        # 点赞数
        noteitem["likes"] = comment_data.get('likes',0)
        yield noteitem

        # 无限调用
        fans_urls = self.fans_url % userid
        follow_urls = self.fllow % userid
        # yield item
        # # 粉丝列表
        yield scrapy.Request(fans_urls, callback=self.fans_list)
        yield scrapy.Request(follow_urls, callback=self.fans_list)
    # 评论详情
    def comment_num(self,response):
        noteitem = response.meta['noteitem']
        nums_list = json.loads(response.text)
        nums_data = nums_list.get('data')
        # 评论数量
        noteitem["comment_count"] = nums_data.get('comment_coun',0)












